# -*- coding: utf-8 -*-
"""
Created on Sat Nov 18 14:46:06 2017

@author: Jan
"""
import redis
import simhash
import logging
import logging.config
import os
import threading
import configparser
from Crawl import Crawl
from concurrent.futures import ThreadPoolExecutor
class DealParser():
    def __init__(self):
        logging.config.fileConfig(os.path.join(os.path.dirname(__file__),'config/logging.conf'))
        self.logger=logging.getLogger('crawljoke')
        self.config=configparser.ConfigParser()
        self.config.read('config/crawl.conf')
        self._lock=threading.Lock()
        pool=redis.ConnectionPool(host=self.config.get('server','host'),port=int(self.config.get('server','port')))
        self.__redis=redis.Redis(connection_pool=pool,charset='utf-8')
        self.__listName=self.config.get('redis','listname')
        self.__tempList=self.config.get('redis','tempjoke')
        self.__parserList=[]
        self.__crawl=Crawl()
        self.__threadnums=self.config.getint('pool','threadnums')
        self.__pool=ThreadPoolExecutor(self.__threadnums)
        #添加默认笑话以减少下面逻辑判断过程
        if self.getListLength() ==0:
            self.__redis.rpush(self.__listName,'所谓殊途同归，讲的是以前所有梦想当飞行员、科学家、大老板……等的中国小朋友们，成年以后的梦想统一变为买房……')
    def addParser(self,parser):
        self.__parserList.append(parser)
    def dealAllJoke(self):
        while self.__parserList and len(self.__parserList) >0:
            parser=self.__parserList.pop()
            self.dealSingleJoke(parser)
    def dealSingleJoke(self,parser):
        next_url=self.dealParserJoke(parser)
        while next_url:
            self.logger.info('下一页：'+str(next_url))
            data=self.__crawl.crawlHtml(next_url)
            parser.addData(data)
            temp_next_url=self.dealParserJoke(parser)
            #判断是否爬取相同的连接，相同连接退出
            if temp_next_url==next_url:
                break
            else:
                next_url=temp_next_url
    def dealParserJoke(self,parser):
        data=None
        try:
            data=parser.paserHtml()
        except Exception as e:
            self.logger.exception('paserHtml报错')
        if data:
            jokes=data.get('jokes')
            #用队列解决并发问题，或者用消息队列...
            if jokes:
                for joke in jokes:
                    #1.redis存储simhash值，与里面每一个进行比较
                    #2.海明距离>3存入redis,mongodb
                    dealJoke=joke.text.strip()
                    if len(dealJoke)>40:
                        #先存入redis，用队列再处理数据
                        self.saveTmpDataToRedis(dealJoke)
            #处理下一页，下一页链接直接保存到redis
            next_url=data.get('next_url')
            return next_url
    def saveTmpDataToRedis(self,joke):
        self.__redis.rpush(self.__tempList,joke)
    def getListLength(self):
        length=self.__redis.llen(self.__listName)
        return length
    def dealRedisData(self):
        tmpJoke=self.__redis.lpop(self.__tempList).decode('utf-8')
        while tmpJoke:
            self._existFlag=False
            self.loopRedis(tmpJoke)
            tmpJoke=self.__redis.lpop(self.__tempList).decode('utf-8')
    def loopRedis(self,joke):
        jokeList=None
        nums=self.getListLength()
        #遍历的块数
        pieceNums=nums//self.__threadnums
        iterNums=self.__threadnums
        print('pieceNums:',pieceNums,' iterNums:',iterNums,' nums',nums)
        #多个线程simhash joke,判断一个线程处理多少条joke
        list_flag=[]
        for num in range(iterNums+1):
            if num==0:
                jokeList=self.__redis.lrange(self.__listName,0,pieceNums)
            else:
                jokeList=self.__redis.lrange(self.__listName,num*pieceNums+1,(num+1)*pieceNums)
            list_flag.append(self.__pool.submit(self.pieceDealJokes,joke,jokeList).result())
        #全是False,说明joke没有重复
        if list_flag.count(True)==0:
            self.__redis.lpush(self.__listName,joke)
    def pieceDealJokes(self,joke,jokeList):
        for item in jokeList:
            redisJoke=item.decode('utf-8')
            if simhash.Simhash(joke).distance(simhash.Simhash(redisJoke))<3:
                #存在类似文档直接返回
                return True
        return False            